#proevolytx@gmail.com
#2012/9/13

from string import *

class Lexer:
    def __init__(s, text, keywords, operators):
        s.text = text
        s.keywords = keywords
        s.operators = operators
        s.index = 0
        
    def ScanAll(s):
        result = []
        while True:
            token = s.ScanNext()
            if token is None:
                break
            else:
                result.append(token)
        return result
                
    def ScanNext(s):
        if s.index >= len(s.text):
            return
        while s.index < len(s.text) and s.text[s.index].isspace():
            s.index += 1
        
        if s.text[s.index].isalpha():
            token = ""
            while s.index < len(s.text) and s.text[s.index].isalnum():
                token += s.text[s.index]
                s.index += 1
            if token in s.keywords:
                return (token, )
            else:
                return ("token", token)
        elif s.text[s.index].isdigit():
            num = 0
            while s.index < len(s.text) and s.text[s.index].isdigit():
                num = num * 10 + int(s.text[s.index])
                s.index += 1
            return ("num", num)
        else:
            for op in s.operators:
                if s.text[s.index : s.index + len(op)] == op:
                    s.index += len(op)
                    return ("op", op)
            raise Exception("Unexpected character " + s.text[s.index])

if __name__ == '__main__':
    text = "for(i=0; i<5; i=i+1) print(i);"
    keywords = ["for", "if"]
    operators = ["+", "-", "*", "/", "<=", "<", ">=", ">", "(", ")", "=", ";"]
    test = Lexer(text, keywords, operators)
    print test.ScanAll()

